import scrapy
from fruiter.items import FruiterItem
import datetime
import re
class FruitSpider(scrapy.Spider):
    name = 'fruit'
    allowed_domains = ['www.sgzmt.com']
    start_urls = ['http://www.sgzmt.com']
    dateTanp='{0:%m%y%d%H%M%S}'.format(datetime.datetime.now())
    dateTamp=int(dateTanp) 
    n=1

    def parse(self, response):#获取水果名称集合链接
        meta={}
        urll_list=response.xpath('/html/body/div/div/div[2]/div/div[1]/ul/li')
        for urlls in urll_list:
            urll= urlls.xpath('.//a/text()').extract_first()
            urll=self.start_urls[0]+'/tag/'+urll+"/"
            yield scrapy.http.Request(url=urll,callback=self.parse_title,meta={'n':1,'urls':urll})
    
    def parse_title(self,response):#获取每个水果各个文章的标题
        n=response.meta['n']
        urll=response.meta['urls']
        meta={}
        
        self.n=self.n+1
        fname=response.xpath('/html/body/div/div/div[1]/div[1]/a[2]/text()').extract_first()
        fruit_list = response.xpath('//*[@id="news"]/div')
        for fruitT in fruit_list:
            furitD=fruitT.xpath('.//h2/a/@href').extract_first()
            furitD=self.start_urls[0]+furitD
            yield scrapy.http.Request(url=furitD,callback=self.parse_deatail,meta={'fname':fname,'pid':self.dateTamp+self.n})
        n=n+1
        nl=0
        for s in response.xpath('/html/body/div/div/div[1]/nav/a'):
            nl=nl+1
        #翻页
        if n<nl-3:
            yield scrapy.http.Request(url=urll+'?page='+str(n),callback=self.parse_title,meta={'n':n,'urls':urll})
     

    def parse_deatail(self,response):#获取各个水果子标题的内容
        item=FruiterItem()
        item['art_name']=response.meta['fname']
        item['art_pid'] =response.meta['pid']
        item['art_chapte']=response.xpath('/html/body/div/div/div[1]/div[2]/h1/text()').extract_first()
        ffrom=response.xpath('/html/body/div/div/div[1]/div[2]/div[1]/p/span[3]').extract_first()
        pat_f='<span>来源：(.*?)</span>'
        item['art_from'] = re.findall(pat_f, ffrom)[0]
        item['art_time']=response.xpath('/html/body/div/div/div[1]/div[2]/div[1]/p/text()').extract_first().replace('\n','').replace('\r','')
        fauthor=response.xpath('/html/body/div/div/div[1]/div[2]/div[1]/p/span[2]').extract_first()
        pat_a='<span>作者：(.*?)</span>'
        item['art_author']=re.findall(pat_a, fauthor)[0]
        contents=response.xpath('/html/body/div/div/div[1]/div[2]/div[2]')
        conts = contents.xpath('string(.)').extract() 
        art_cont=''
        for cont in conts:
            cont = cont.replace('\n','').replace('\r','').replace('\xa0','').replace('\t','').replace('\u3000','')
            art_cont=art_cont+cont   
        item['art_content']=art_cont.strip()
        yield item
        